Source code for src.cmd.pre_process

from util import segment_run_map
from ..util import check_file
from ..configuration import config, dir_map, db_info

def pre_process(cmd):
    print "Start PreProcessing..."
    
    args = cmd.args
    _segments = [segment_run_map[seg] for seg in args.segments.split('|')]
    
    #validate user data
    opt = {'out_dir': check_file(config.out_dir, as_dir = True),
           'meta_dir': check_file(config.meta_dir, as_dir = True),
           'rules': dir_map['pre_process_rules_dir'],
           'out_format': 'mysql',
           'out_delimiter': '|',
           'verbose': args.verbose,
           'db_info': db_info['stand'],
           'limit': args.limit,
           'halt_on_err': config.debug_halt_on_error,
           'phase_id': '0',
    }
    
    for segment in sorted(_segments):    
        if segment in ('sentence', 'supervision'): #removed supervision from pre_processing
            continue
        _opt = opt.copy()
        _opt['segment'] = segment
        cmd.task_queue.put(_opt)
    
    cmd.start() #start the arrest, demographic, and supervision workers
    cmd.task_queue.join() #wait for the workers to finish
    
    opt['segment'] = 'sentence'   
    cmd.task_queue.put(opt)
    cmd.start() #start the sentence worker
    cmd.task_queue.join() #wait for the workers to finish
    cmd.flush() #flush the cmd logs